package aiingames.minimax;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import aiingames.minimax.State.Player;


public class TDTicTacToeLearner {
	
	Random rand = new Random();

	private double alpha = 0.1;
	
	private double epsilon = 0.001;
	
	private Map<String,Double> lookupTable = new HashMap<String,Double>();
	
	private State previous = null;

	private final double valSign;

	
	public TDTicTacToeLearner(Player p) {
		this.valSign = p == Player.MAX ? 1 : -1;
	}
	
	public void informAboutGameResult(State terminal) {
		if (terminal.utility() == this.valSign) {
			this.previous = null;
			return;
		}
		
		if (this.lookupTable.get(terminal.toString()) == null) {
			createLookUpTableEntry(terminal);
		}
		backupPreviousState(terminal);
		this.previous = null;
	}
	
	
	public State getTDTicTacToeLearnerDecission(State currentState) {
						
		if (Math.random() < this.epsilon) {
			State nextState = getNextStateExploratory(currentState);
			this.previous = nextState;
			if (lookupTable.get(nextState.toString())==null) {
				createLookUpTableEntry(nextState);
			}
			
			return nextState;
		}
				
		State nextState = getNextStateGreedily(currentState);
		
		backupPreviousState(nextState);
		
		
		this.previous = nextState;
		return nextState;
	}


	private double createLookUpTableEntry(State nextState) {
		double nextStateVal;
		if (nextState.terminalTest()) {
			nextStateVal = this.valSign * nextState.utility();
		} else {
			nextStateVal = 2*this.rand.nextDouble() -1;
		}
		this.lookupTable.put(nextState.toString(), nextStateVal);
		
		return nextStateVal;
	}

	private void backupPreviousState(State nextState) {
		if (previous != null) {
			double prev = this.lookupTable.get(this.previous.toString());
			double next = this.lookupTable.get(nextState.toString());
			double backedUp = prev + this.alpha * (next - prev);
			this.lookupTable.put(this.previous.toString(), backedUp);
		}
		
	}


	private State getNextStateExploratory(State currentState) {
		List<Action> l = currentState.getActions();
		int idx = this.rand.nextInt(l.size());
		return l.get(idx).getResult(currentState);
	}


	private State getNextStateGreedily(State currentState) {
		List<Action> l = currentState.getActions();
		
		
		
		double bestVal = Double.NEGATIVE_INFINITY;
		State bestNextState = null;
		
		for (Action  a : l) {
			State tmp = a.getResult(currentState);
			Double tmpVal = this.lookupTable.get(tmp.toString());
			if (tmpVal == null) {
				tmpVal = createLookUpTableEntry(tmp);
			}
			if (tmpVal > bestVal) {
				bestVal = tmpVal;
				bestNextState  = tmp;
			}
		}
		return bestNextState;
	}
	
	public String toString() {
		return "Size of lookup table is:" + this.lookupTable.size();
	}
}
